import requests,time,pymongo
from bs4 import BeautifulSoup
client = pymongo.MongoClient('localhost',27017)
five_eight_web = client['five_eight_web']
sj_all_links = five_eight_web['sj_all_links']
headers = {
    'Cookie':'citydomain=bj; ganji_uuid=4833701062167349196422; _gl_tracker=%7B%22ca_source%22%3A%22-%22%2C%22ca_name%22%3A%22-%22%2C%22ca_kw%22%3A%22-%22%2C%22ca_id%22%3A%22-%22%2C%22ca_s%22%3A%22self%22%2C%22ca_n%22%3A%22-%22%2C%22ca_i%22%3A%22-%22%2C%22sid%22%3A51296451069%7D; __utma=32156897.1107777287.1525241697.1525241697.1525241697.1; __utmc=32156897; __utmz=32156897.1525241697.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); GANJISESSID=960n193u2r9jra75dof3k0c0fi; ganji_xuuid=141a791f-d43f-4c4a-9c4e-35217cce2ddc.1525241776606; xxzl_deviceid=qcrs8aaRoH%2F%2B3oG%2BqzHO%2F4Xd%2BEQqaAz8Zh8Atzis1yBNgOVW7hdtp9587DHV3A6H; 58uuid=ca04234d-1f02-47dd-954f-0832a3d10bc6; init_refer=; new_uv=1; new_session=0; als=0; lg=1; ganji_login_act=1525243589982; __utmb=32156897.7.10.1525241697',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36'
}
# http://bj.ganji.com/shouji/o2/
def get_all_link(url,page):
    urls = '{}/o{}/'.format(url,str(page))
    time.sleep(3)
    web_data = requests.get(urls,headers=headers)
    soup = BeautifulSoup(web_data.text,'lxml')
    link = soup.select('a.t')
    for i in link:
        links = i.get('href').split('?')[0]
        sj_all_links.insert_many(links)
        print(links)
for i in range(0,21):
    get_all_link('http://bj.ganji.com/shouji',i)
# get_all_link('http://bj.ganji.com/shouji',1)
